package com.doitedu.mr.day07.skew;

import com.doitedu.mr.day03.WordCountMapper;
import com.doitedu.mr.day03.WordCountReducer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.IOException;

/**
 * @Date 2021/12/9
 * @Created by HANGGE
 * @Description TODO
 */
public class SkewDemo {
    static  class  SkewDemoMapper extends Mapper<LongWritable, Text, Text, IntWritable> {
        Text k = new Text() ;
        IntWritable v = new IntWritable(1);
        @Override
        protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
            String line = value.toString();
            String[] words = line.split("\\s+");
            for (String word : words) {
                k.set(word);
                context.write(k,v);
            }
        }
    }
    static  class  SkewDemoReducer extends Reducer<Text , IntWritable , Text, IntWritable> {
        @Override
        protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws IOException, InterruptedException {
            for (IntWritable value : values) {
                context.write(key , value);
            }
        }
    }

    public static void main(String[] args) throws  Exception{

        // Job  统计文件中单词出现的次数
        Configuration conf = new Configuration();
        // 1  创建 JOB
        Job job = Job.getInstance(conf, "skew");
        // 1) Mapper 类
        job.setMapperClass(SkewDemoMapper.class);
        // 2) Reducer类
        job.setReducerClass(SkewDemoReducer.class);
        // 3)  设置map输出的数据类型
        job.setMapOutputKeyClass(Text.class);
        job.setMapOutputValueClass(IntWritable.class);

        // 4)  设置reduce输出的数据类型
        job.setOutputKeyClass(Text.class);
        job.setOutputValueClass(IntWritable.class);
        // 设置两个reducetask
        job.setNumReduceTasks(2);

        // 5) 设置输入的数据路径
        FileInputFormat.setInputPaths(job , new Path("E:\\mrdata\\skew\\input"));
        // 6) 设置输出的结果保存路径
        FileOutputFormat.setOutputPath(job, new Path("E:\\mrdata\\skew\\out3"));
        // 2 提交工作
        // 等待执行完成 , 打印执行过
        job.waitForCompletion(true) ;

    }
}
